From: iap10@labyrinth.cl.cam.ac.uk Date: Thu, 20 May 2004 17:43:17 +0000 (+0000) Subject: bitkeeper revision 1.911.1.5 (40acee35tmC494sW3OcQEIN4ymTS1w) X-Git-Tag: archive/raspbian/4.8.0-1+rpi1~1^2~18204^2 X-Git-Url: https://dgit.raspbian.org/%22http:/www.example.com/cgi/%22https:/%22bookmarks://%22/%22http:/www.example.com/cgi/%22https:/%22bookmarks:/%22?a=commitdiff_plain;h=6bf53ef08d47db840c3d18ed5eb0e1fcf1dda730;p=xen.git bitkeeper revision 1.911.1.5 (40acee35tmC494sW3OcQEIN4ymTS1w) Nasty temporary mechanism that allows dom0 to block after issuing a stop on another domain. --- diff --git a/tools/xc/lib/xc_domain.c b/tools/xc/lib/xc_domain.c index 6d0dd6d0f3..97b2b26f6d 100644 --- a/tools/xc/lib/xc_domain.c +++ b/tools/xc/lib/xc_domain.c @@ -46,6 +46,7 @@ int xc_domain_stop(int xc_handle, dom0_op_t op; op.cmd = DOM0_STOPDOMAIN; op.u.stopdomain.domain = (domid_t)domid; + op.u.stopdomain.sync = 0; // async return do_dom0_op(xc_handle, &op); } diff --git a/tools/xc/lib/xc_linux_save.c b/tools/xc/lib/xc_linux_save.c index ddcdb1df39..857007fd3e 100644 --- a/tools/xc/lib/xc_linux_save.c +++ b/tools/xc/lib/xc_linux_save.c @@ -82,6 +82,67 @@ inline void set_bit ( int nr, volatile void * addr) (1 << (nr % (sizeof(unsigned long)*8) ) ); } +long long tv_to_us( struct timeval *new ) +{ + return (new->tv_sec * 1000000) + new->tv_usec; +} + +long long tvdelta( struct timeval *new, struct timeval *old ) +{ + return ((new->tv_sec - old->tv_sec)*1000000 ) + + (new->tv_usec - old->tv_usec); +} + +int track_cpu_usage_dom0( int xc_handle, int print ) +{ + static struct timeval wall_last; + static long long cpu_last; + + struct timeval wall_now; + long long cpu_now, wall_delta, cpu_delta; + + gettimeofday(&wall_now, NULL); + + cpu_now = xc_domain_get_cpu_usage( xc_handle, 0 )/1000; + + wall_delta = tvdelta(&wall_now,&wall_last)/1000; + cpu_delta = (cpu_now - cpu_last)/1000; + + if(print) + printf("Dom0 : wall delta %lldms, cpu delta %lldms : %d%%\n", + wall_delta, cpu_delta, (cpu_delta*100)/wall_delta); + + cpu_last = cpu_now; + wall_last = wall_now; + + return 0; +} + +int track_cpu_usage_target( int xc_handle, u64 domid, int print ) +{ + static struct timeval wall_last; + static long long cpu_last; + + struct timeval wall_now; + long long cpu_now, wall_delta, cpu_delta; + + gettimeofday(&wall_now, NULL); + + cpu_now = xc_domain_get_cpu_usage( xc_handle, domid )/1000; + + wall_delta = tvdelta(&wall_now,&wall_last)/1000; + cpu_delta = (cpu_now - cpu_last)/1000; + + if(print) + printf("Target: wall delta %lldms, cpu delta %lldms : %d%%\n", + wall_delta, cpu_delta, (cpu_delta*100)/wall_delta); + + cpu_last = cpu_now; + wall_last = wall_now; + + return 0; +} + int xc_linux_save(int xc_handle, u64 domid, @@ -95,10 +156,11 @@ int xc_linux_save(int xc_handle, int verbose = flags & XCFLAGS_VERBOSE; int live = flags & XCFLAGS_LIVE; int debug = flags & XCFLAGS_DEBUG; - int sent_last_iter, sent_this_iter, skip_this_iter, max_iters; - - /* Remember if we stopped the guest, so we can restart it on exit. */ - int we_stopped_it = 0; + int sent_last_iter, sent_this_iter, skip_this_iter; + + /* Important tuning parameters */ + int max_iters = 29; // limit us to 30 times round loop + int max_factor = 3; // never send more than 3x nr_pfns /* The new domain's shared-info frame number. */ unsigned long shared_info_frame; @@ -153,39 +215,16 @@ int xc_linux_save(int xc_handle, } /* Ensure that the domain exists, and that it is stopped. */ - for ( ; ; ) - { - op.cmd = DOM0_GETDOMAININFO; - op.u.getdomaininfo.domain = (domid_t)domid; - op.u.getdomaininfo.ctxt = &ctxt; - if ( (do_dom0_op(xc_handle, &op) < 0) || - ((u64)op.u.getdomaininfo.domain != domid) ) - { - PERROR("Could not get info on domain"); - goto out; - } - - memcpy(name, op.u.getdomaininfo.name, sizeof(name)); - shared_info_frame = op.u.getdomaininfo.shared_info_frame; - if ( op.u.getdomaininfo.state == DOMSTATE_STOPPED ) - break; - - we_stopped_it = 1; - - op.cmd = DOM0_STOPDOMAIN; - op.u.stopdomain.domain = (domid_t)domid; - if ( do_dom0_op(xc_handle, &op) != 0 ) - { - we_stopped_it = 0; - PERROR("Stopping target domain failed"); - goto out; - } - - usleep(1000); // 1ms - printf("Sleep for 1ms\n"); + if ( xc_domain_stop_sync( xc_handle, domid, &op, &ctxt ) ) + { + PERROR("Could not sync stop domain"); + goto out; } + memcpy(name, op.u.getdomaininfo.name, sizeof(name)); + shared_info_frame = op.u.getdomaininfo.shared_info_frame; + /* A cheesy test to see whether the domain contains valid state. */ if ( ctxt.pt_base == 0 ) { @@ -292,7 +331,6 @@ int xc_linux_save(int xc_handle, last_iter = 0; sent_last_iter = 1<<20; // 4GB's worth of pages - max_iters = 29; // limit us to 30 times round loop } else last_iter = 1; @@ -384,8 +422,11 @@ int xc_linux_save(int xc_handle, goto out; } - /* Now write out each data page, canonicalising page tables as we go... */ + track_cpu_usage_dom0(xc_handle, 0); + track_cpu_usage_target( xc_handle, domid, 0); + /* Now write out each data page, canonicalising page tables as we go... */ + while(1) { unsigned int prev_pc, batch, sent_this_iter; @@ -615,6 +656,10 @@ int xc_linux_save(int xc_handle, verbose_printf("\b\b\b\b100%% (pages sent= %d, skipped= %d )\n", sent_this_iter, skip_this_iter ); + + track_cpu_usage_dom0(xc_handle, 1); + track_cpu_usage_target( xc_handle, domid, 1); + if ( last_iter ) { @@ -649,12 +694,12 @@ int xc_linux_save(int xc_handle, // ( sent_this_iter > (sent_last_iter * 0.95) ) || (iter >= max_iters) || (sent_this_iter+skip_this_iter < 10) || - (total_sent > nr_pfns*2) ) + (total_sent > nr_pfns*max_factor) ) { DPRINTF("Start last iteration\n"); last_iter = 1; - xc_domain_stop_sync( xc_handle, domid ); + xc_domain_stop_sync( xc_handle, domid, &op, NULL ); } diff --git a/tools/xc/lib/xc_private.c b/tools/xc/lib/xc_private.c index 430dc6ec11..d97ba37c3d 100644 --- a/tools/xc/lib/xc_private.c +++ b/tools/xc/lib/xc_private.c @@ -376,51 +376,67 @@ int finish_mmu_updates(int xc_handle, mmu_t *mmu) /* this function is a hack until we get proper synchronous domain stop */ -int xc_domain_stop_sync( int xc_handle, domid_t domid ) +int xc_domain_stop_sync( int xc_handle, domid_t domid, + dom0_op_t *op, full_execution_context_t *ctxt) { - dom0_op_t op; int i; - - - op.cmd = DOM0_STOPDOMAIN; - op.u.stopdomain.domain = (domid_t)domid; - if ( do_dom0_op(xc_handle, &op) != 0 ) - { - PERROR("Stopping target domain failed"); - goto out; - } - usleep(100); // 100us + printf("Sleep:"); for(i=0;;i++) - { - if (i>0) - if (i==1) printf("Sleep."); - else printf("."); - - op.cmd = DOM0_GETDOMAININFO; - op.u.getdomaininfo.domain = (domid_t)domid; - op.u.getdomaininfo.ctxt = NULL; - if ( (do_dom0_op(xc_handle, &op) < 0) || - ((u64)op.u.getdomaininfo.domain != domid) ) + { + + op->cmd = DOM0_STOPDOMAIN; + op->u.stopdomain.domain = (domid_t)domid; + op->u.stopdomain.sync = 1; + do_dom0_op(xc_handle, op); + /* can't trust return code due to sync stop hack :-(( */ + + + op->cmd = DOM0_GETDOMAININFO; + op->u.getdomaininfo.domain = (domid_t)domid; + op->u.getdomaininfo.ctxt = ctxt; + if ( (do_dom0_op(xc_handle, op) < 0) || + ((u64)op->u.getdomaininfo.domain != domid) ) { PERROR("Could not get info on domain"); goto out; } - if ( op.u.getdomaininfo.state == DOMSTATE_STOPPED ) + if ( op->u.getdomaininfo.state == DOMSTATE_STOPPED ) { printf("Domain %lld stopped\n",domid); return 0; } - usleep(1000); + printf("."); + + //usleep(1000); } + printf("\n"); + out: return -1; } +long long xc_domain_get_cpu_usage( int xc_handle, domid_t domid ) +{ + dom0_op_t op; + + op.cmd = DOM0_GETDOMAININFO; + op.u.getdomaininfo.domain = (domid_t)domid; + op.u.getdomaininfo.ctxt = NULL; + if ( (do_dom0_op(xc_handle, &op) < 0) || + ((u64)op.u.getdomaininfo.domain != domid) ) + { + PERROR("Could not get info on domain"); + return -1; + } + return op.u.getdomaininfo.cpu_time; +} + + /**********************************************************************/ // this is shared between save and restore, and may be useful. diff --git a/tools/xc/lib/xc_private.h b/tools/xc/lib/xc_private.h index e3eff85e59..330edca50f 100644 --- a/tools/xc/lib/xc_private.h +++ b/tools/xc/lib/xc_private.h @@ -248,6 +248,9 @@ void * mfn_mapper_queue_entry(mfn_mapper_t *t, int offset, /*********************/ -int xc_domain_stop_sync( int xc_handle, domid_t dom ); +int xc_domain_stop_sync( int xc_handle, domid_t dom, + dom0_op_t *op, full_execution_context_t *ctxt ); + +long long xc_domain_get_cpu_usage( int xc_handle, domid_t domid ); #endif /* __XC_PRIVATE_H__ */ diff --git a/xen/common/dom0_ops.c b/xen/common/dom0_ops.c index 3b413a1faa..780e76e638 100644 --- a/xen/common/dom0_ops.c +++ b/xen/common/dom0_ops.c @@ -102,6 +102,20 @@ long do_dom0_op(dom0_op_t *u_dom0_op) case DOM0_STOPDOMAIN: { ret = stop_other_domain(op->u.stopdomain.domain); + + /* This is grim, but helps for live migrate. It's also unsafe + in the strict sense as we're not explicitly setting a + timeout, but dom0 is bound to have other timers going off to + wake us back up. + We go to sleep so that the other domain can stop quicker, hence + we have less total down time in a migrate. + */ + if( ret == 0 && op->u.stopdomain.sync == 1 ) + { + extern long do_block( void ); + printk("T\n"); + do_block(); // Yuk... + } } break; diff --git a/xen/common/domain.c b/xen/common/domain.c index ee11f20fcd..52becabf5c 100644 --- a/xen/common/domain.c +++ b/xen/common/domain.c @@ -266,6 +266,16 @@ void stop_domain(void) unlazy_fpu(current); wmb(); /* All CPUs must see saved info in state TASK_STOPPED. */ set_current_state(TASK_STOPPED); + + /* OK, this is grim, but helps speed up live migrate. When a domain stops, + kick Dom0 */ + { + struct task_struct *p; + printk("S\n"); + guest_schedule_to_run( p = find_domain_by_id(0ULL) ); + put_task_struct(p); + } + __enter_scheduler(); } diff --git a/xen/common/schedule.c b/xen/common/schedule.c index 5aa12bf8cb..b5e4219d6c 100644 --- a/xen/common/schedule.c +++ b/xen/common/schedule.c @@ -27,6 +27,9 @@ #include #include #include + +#undef TRACE_BUFFER + #include /*#define WAKEUP_HISTO*/ @@ -216,7 +219,7 @@ void wake_up(struct task_struct *p) /* * Block the currently-executing domain until a pertinent event occurs. */ -static long do_block(void) +long do_block(void) { ASSERT(current->domain != IDLE_DOMAIN_ID); current->shared_info->vcpu_data[0].evtchn_upcall_mask = 0; diff --git a/xen/include/hypervisor-ifs/dom0_ops.h b/xen/include/hypervisor-ifs/dom0_ops.h index 58b1480525..2a17605bf2 100644 --- a/xen/include/hypervisor-ifs/dom0_ops.h +++ b/xen/include/hypervisor-ifs/dom0_ops.h @@ -74,6 +74,9 @@ typedef struct dom0_stopdomain_st { /* IN parameters. */ domid_t domain; + /* hack to indicate that you want to wait for other domain -- replace + with proper sychronous stop soon! */ + int sync; } dom0_stopdomain_t; #define DOM0_GETDOMAININFO 12